from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import gc
import math
import os
import random
import sys
import time
from sets import Set
import numpy as np

# ONLY ONE variable!
termIDAndPrunedListLengthDict = {}
print "Program Begins..."
# ONLY for gov2
TOTAL_NUM_OF_POSTINGS = 6451948010
numOfPostingNeededToBePoppedAtDebugPercentage = 5000000
numOfPostingNeededToBePoppedAt1Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.01)
numOfPostingNeededToBePoppedAt2Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.02)
numOfPostingNeededToBePoppedAt3Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.03)
numOfPostingNeededToBePoppedAt4Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.04)
numOfPostingNeededToBePoppedAt5Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.05)
numOfPostingNeededToBePoppedAt6Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.06)
numOfPostingNeededToBePoppedAt7Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.07)
numOfPostingNeededToBePoppedAt8Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.08)
numOfPostingNeededToBePoppedAt9Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.09)
numOfPostingNeededToBePoppedAt10Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.1)
numOfPostingNeededToBePoppedAt15Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.15)
numOfPostingNeededToBePoppedAt20Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.2)
numOfPostingNeededToBePoppedAt30Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.3)
numOfPostingNeededToBePoppedAt40Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.4)
numOfPostingNeededToBePoppedAt50Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.5)
numOfPostingNeededToBePoppedAt60Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.6)
numOfPostingNeededToBePoppedAt70Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.7)
numOfPostingNeededToBePoppedAt80Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.8)
numOfPostingNeededToBePoppedAt90Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.9)
numOfPostingNeededToBePoppedAt100Percentage = int(TOTAL_NUM_OF_POSTINGS * 1.0)

print "numOfPostingNeededToBePoppedAtDebugPercentage:",numOfPostingNeededToBePoppedAtDebugPercentage
print "numOfPostingNeededToBePoppedAt1Percentage:",numOfPostingNeededToBePoppedAt1Percentage
print "numOfPostingNeededToBePoppedAt2Percentage:",numOfPostingNeededToBePoppedAt2Percentage
print "numOfPostingNeededToBePoppedAt3Percentage:",numOfPostingNeededToBePoppedAt3Percentage
print "numOfPostingNeededToBePoppedAt4Percentage:",numOfPostingNeededToBePoppedAt4Percentage
print "numOfPostingNeededToBePoppedAt5Percentage:",numOfPostingNeededToBePoppedAt5Percentage
print "numOfPostingNeededToBePoppedAt6Percentage:",numOfPostingNeededToBePoppedAt6Percentage
print "numOfPostingNeededToBePoppedAt7Percentage:",numOfPostingNeededToBePoppedAt7Percentage
print "numOfPostingNeededToBePoppedAt8Percentage:",numOfPostingNeededToBePoppedAt8Percentage
print "numOfPostingNeededToBePoppedAt9Percentage:",numOfPostingNeededToBePoppedAt9Percentage
print "numOfPostingNeededToBePoppedAt10Percentage:",numOfPostingNeededToBePoppedAt10Percentage
print "numOfPostingNeededToBePoppedAt15Percentage:",numOfPostingNeededToBePoppedAt15Percentage
print "numOfPostingNeededToBePoppedAt20Percentage:",numOfPostingNeededToBePoppedAt20Percentage
print "numOfPostingNeededToBePoppedAt30Percentage:",numOfPostingNeededToBePoppedAt30Percentage
print "numOfPostingNeededToBePoppedAt40Percentage:",numOfPostingNeededToBePoppedAt40Percentage
print "numOfPostingNeededToBePoppedAt50Percentage:",numOfPostingNeededToBePoppedAt50Percentage
print "numOfPostingNeededToBePoppedAt60Percentage:",numOfPostingNeededToBePoppedAt60Percentage
print "numOfPostingNeededToBePoppedAt70Percentage:",numOfPostingNeededToBePoppedAt70Percentage
print "numOfPostingNeededToBePoppedAt80Percentage:",numOfPostingNeededToBePoppedAt80Percentage
print "numOfPostingNeededToBePoppedAt90Percentage:",numOfPostingNeededToBePoppedAt90Percentage
print "numOfPostingNeededToBePoppedAt100Percentage:",numOfPostingNeededToBePoppedAt100Percentage

numOfBytes = 0
numOfPostingPopped = 0
fileCounter = 0
if len(sys.argv) != 2:
    print "USAGE:","program name allPostingFile"
inputFileName = sys.argv[1]
inputFileHandler = open(inputFileName,"rb")
print "--->posting file to evaluate:",inputFileName
statinfo = os.stat(inputFileName)
FILE_SIZE = statinfo.st_size
print "FILE_SIZE:",FILE_SIZE
while numOfBytes < FILE_SIZE:
    # current version
    # each time, read the info of 1M postings
    byteStringBuffer = inputFileHandler.read( 1000000 * 16)
    byteStringBufferIndexPosition = 0
    for i in range(0,1000000):
        byteString = byteStringBuffer[byteStringBufferIndexPosition:byteStringBufferIndexPosition+16]
        byteStringBufferIndexPosition += 16
        (termID,docID,currentProbability,impactScore) = unpack( "2I2f", byteString)
        if termID not in termIDAndPrunedListLengthDict:
            termIDAndPrunedListLengthDict[termID] = 1
        else:
            termIDAndPrunedListLengthDict[termID] += 1
        numOfPostingPopped += 1
        if numOfPostingPopped == numOfPostingNeededToBePoppedAtDebugPercentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt1Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt2Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt3Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt4Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt5Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt6Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt7Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt8Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt9Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt10Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt15Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt20Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt30Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt40Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt50Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt60Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt70Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt80Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt90Percentage:
            
            outputfileName = inputFileName + "_termAndPrunedListLength20140720" + "_" + str(fileCounter)
            outputfineHandler = open(outputfileName,"w")
            for termID in termIDAndPrunedListLengthDict:
                outputfineHandler.write(str(termID) + " " + str(termIDAndPrunedListLengthDict[termID]) + "\n")
            outputfineHandler.close()
            print "OVERALL:"
            print "len(termIDAndPrunedListLengthDict):",len(termIDAndPrunedListLengthDict)
            print "outputfileName:",outputfileName
            print "numOfPostingPopped:",numOfPostingPopped
            fileCounter += 1
        numOfBytes += 12
        if numOfPostingPopped % 1000000 == 0:
            print str(numOfPostingPopped),"postings have been examined."

inputFileHandler.close()
print "inputFileName0:",inputFileName0
print "inputFileName1:",inputFileName1
print "program ends."